# coding=utf-8
import os
import time
import json
import codecs
import gevent
import requests

from gevent import monkey
from tools import save_file, channel_1, field_handle, aes_2
from concurrent.futures import ThreadPoolExecutor, wait
from utils.value_formater import is_valid_idnum, is_valid_name

monkey.patch_all()

requests.packages.urllib3.disable_warnings()

class_name = "jiyouxiaozhushou"

appid = "wx83fd00e39502c5a7"

headers = {
    "Host": "wx.dafysz.cn",
    "Connection": "keep-alive",
    "Content-Length": "92",
    "Accept": "application/json, text/plain, */*",
    "Content-Type": "application/json;charset=UTF-8",
    "Origin": "https://wx.dafysz.cn",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat QBCore/3.43.691.400 QQBrowser/9.0.2524.400",
    "Referer": "https://wx.dafysz.cn/myDafy/manage/bind?redirect=&state=",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.8,en-us;q=0.6,en;q=0.5;q=0.4",
}

field_name_mapper = {
    u"": "",
}

query_url = "https://wx.dafysz.cn/wechat-web/bind/getPhoneForBind"
token_query_url = "http://127.0.0.1:7070/get?appid={}".format(appid)

raw_file_path = ur"D:\催收平台的爬取数据\微信公众号_既有小助手/raw_data.txt"
# raw_file_path = ur"D:\催收平台的爬取数据\微信公众号_既有小助手/raw_data_example.txt"
processed_file_path = ur"D:\催收平台的爬取数据\微信公众号_既有小助手/processed_data.txt"

save_file_name = "jiyouxiaozhushou"

worker_num = 10
crawl_num = 0

crawled_set = set()

file_item_header = "query_id_number^query_name^query_province^query_city^phone^person_id^crawled_time"

item_header = "phone^crawled_time"


def fill_token(payload):
    content = get_content(url=token_query_url)
    if content != "":
        json_content = json.loads(content)

        if json_content.get("code", 504) == 200:
            token = json_content.get("data", {}).get("token", "")

            payload["token"] = token

    return payload


def get_content(url, headers=None, payload=None, method="GET"):
    while 1:
        if "127.0.0.1" not in url and payload is not None:
            payload = fill_token(payload)

        try:
            with gevent.Timeout(10, requests.Timeout):
                if method == "GET":
                    response = requests.get(url, headers=headers, params=payload, verify=False, allow_redirects=False)
                else:
                    response = requests.post(url, headers=headers, json=payload, verify=False, allow_redirects=False)
                response.encoding = "utf-8"

                if response.status_code in (302, 200):
                    return response.text
                elif response.status_code in (404, 500):
                    return ""
                else:
                    time.sleep(0.5)
        except Exception as e:
            print(e.message)
            continue


def load_crawled():
    print ("load crawled phones...")

    if os.path.exists(raw_file_path):
        with codecs.open(raw_file_path, 'r', encoding="utf-8") as read_file:

            for line in read_file:
                crawled_set.update([json.loads(line).get("query_id_number")])


def crawl(file_path):
    def get_line_generator():
        with codecs.open(file_path, 'r', encoding="utf-8") as read_file:
            for line in read_file:

                try:
                    id_number = line.split(",")[0]
                    if id_number in crawled_set:
                        # crawled_set.remove(id_number)
                        continue
                except Exception as e:
                    print(e)
                    continue

                try:
                    yield line
                except StopIteration:
                    yield []

    def query_by_file(raw_file_path):
        global crawl_num

        try:
            for line in line_generator:

                crawl_num += 1

                print (crawl_num)

                id_number, name, province, city = line.replace("\r", "").replace("\n", "").split(",")

                param_dict = {
                    "query_id_number": id_number,
                    "query_name": name,
                    "query_province": province,
                    "query_city": city,
                }

                try:
                    result = query(id_number, name, province, city)
                except:
                    print (line)
                    continue

                if result:
                    save_item_with_json(raw_file_path, result)

        except StopIteration:
            return

    load_crawled()

    line_generator = get_line_generator()

    pool = ThreadPoolExecutor(max_workers=worker_num)

    executor_list = []

    for _ in xrange(worker_num):
        executor_list.append(pool.submit(query_by_file, raw_file_path))

    print (wait(executor_list))


def process_for_file(result_dict):
    value_lists = process_one(json.dumps(result_dict))

    # header
    # 查询时所用的身份证^查询时所用的姓名^查询时所用的省份^查询时所用的城市^订单代码^地址^宽带账号^宽带密码^爬取时间
    # query_id_number^query_name^query_province^query_city^code^address^account^password^crawled_time

    return [value_lists]


def process_for_channel_1(result_dict):
    phone = result_dict.get(u"phone", "")

    item = result_dict

    value_list = list([
        field_handle(item.get("query_id_number", "")),  # idnumber
        aes_2(field_handle(phone)),                     # c_phone
        u"本人",                                         # relation
        field_handle(item.get("query_name", "")),       # c_name
        "",                                              # comment
        "1",                                             # level
        "",                                              # creatd_time
    ])

    return [value_list]


def process_one_with_dict(line):
    value_list = process_one(line)

    if value_list is not None:
        value_dict = dict(zip(item_header.split("^"), value_list))
    else:
        return ""

    return value_dict


def process_one(line):
    item = json.loads(line)

    phone = item.get(u"phone", "")
    if phone is None or phone.strip() == "":
        return None

    value_list = list()

    value_list.append(item.get("query_id_number"))
    value_list.append(item.get("query_name"))
    value_list.append(item.get("query_province"))
    value_list.append(item.get("query_city"))
    value_list.append(item.get("phone"))
    value_list.append(str(item.get("personId")))
    value_list.append(item.get("crawled_time"))

    return value_list


def process(file_path=None):
    if file_path is None:
        file_path = raw_file_path

    with codecs.open(file_path, 'r', encoding="utf-8") as read_file:
        for line in read_file:

            value_list = process_one(line)

            # header
            # 查询时身份证号^查询时姓名^查询时省份^查询时城市^电话^个人ID^爬取时间
            # query_id_number^query_name^query_province^query_city^phone^person_id^crawled_time
            if value_list is None:
                continue

            save_value_list(processed_file_path, value_list)


def query(idnum, name, province, city):
    result_dict = {
        "query_id_number": idnum,
        "query_name": name,
        "query_province": province,
        "query_city": city,
        "crawled_time": get_current_datetime()
    }

    payload = {
        "ident": idnum,
        "name": name,
    }

    if is_valid_idnum(idnum) is False or is_valid_name(name) is False:
        return result_dict

    content = get_content(query_url, headers=headers, payload=payload, method="POST")

    if content != "":
        if u"不存在" in content:
            return result_dict
        elif u"重新" in content:
            exit(0)

        json_content = json.loads(content)

        if json_content.get("status", "") == "success":
            result_dict.update({"phone": json_content.get("data", {}).get("phone", "")})
            result_dict.update({"personId": json_content.get("data", {}).get("personId", "")})

    return result_dict


def querry(task):
    id_number = task.get("person", {}).get("idnumber", "")
    name = task.get("person", {}).get("name", "")
    province = task.get("person", {}).get("id_province", "")
    city = task.get("person", {}).get("id_city", "")

    result_dict = query(id_number, name, province, city)

    # result = process_one_with_dict(json.dumps(result_dict))

    return result_dict


def querry2(person):
    id_number = person.get("idnumber", "")
    name = person.get("name", "")
    province = person.get("id_province", "")
    city = person.get("id_city", "")

    result_dict = query(id_number, name, province, city)

    result = process_one_with_dict(json.dumps(result_dict))

    return result


def get_current_datetime():
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))


def save_value_list(file_path, value_list, sep="^"):
    for i in xrange(len(value_list)):
        if value_list[i] is None:
            value_list[i] = ""
        if isinstance(value_list[i], int):
            value_list[i] = unicode(value_list[i])
        value_list[i] = value_list[i].replace("\r", "").replace("\n", "").replace("^", "|").strip()

    codecs.open(file_path, 'a', encoding="utf-8").write(sep.join(value_list) + "\n")


def save_item_with_json(file_path, item):
    codecs.open(file_path, 'a', encoding="utf-8").write(json.dumps(item, ensure_ascii=False) + "\n")


def query_wrap(task):
    raw_result = querry(task)

    value_lists = process_for_file(raw_result)
    for value_list in value_lists:
        if value_list:
            value_str = "^".join(value_list)

            save_file(filename=save_file_name, head=file_item_header, file=value_str)

    channel_value_list = process_for_channel_1(raw_result)
    for channel_value in channel_value_list:
        if channel_value:
            value_str = "^".join(channel_value)

            channel_1(value_str)


def main():
    # print (jyxzs.query({"query_id_number": "32088219800305301X", "query_name": u"王少金"}))
    # jyxzs.crawl(u"E:\资源\查询参数/hadoop_name_id_not_null.txt")
    # jyxzs.process()

    # 110105198302190049 吴尘
    # {'jiyouxiaozhushou': {'phone': u'18801095825', 'crawled_time': u'2018-03-27 11:09:19'}}
    # 32088219800305301X 王少金
    # {'jiyouxiaozhushou': {'phone': u'15251622798', 'crawled_time': u'2018-03-27 11:09:55'}}

    # task = {
    #     "taskid": "32088219800305301X",
    #     "priority": "10",
    #     "entrance": ["haushudianxin", "mifenka"],
    #     'person': {
    #         "idnum": "32088219800305301X",
    #         "name": "王少金",
    #         "phone": "1xxxxxxxxxx",
    #         "pwd": "111111",
    #     },
    # }

    task = {
        "taskid": "32088219800305301X",
        "priority": "10",
        "entrance": ["haushudianxin", "mifenka"],
        'person': {
            "idnumber": "110105198302190049",
            "name": u"吴尘",
            "mobile": "1xxxxxxxxxx",
            "id_province": u"北京市",
            "id_city": u"北京市",
            "mo_province": "",
            "mo_city": "",
            "operator": ""
        },
    }

    # 如果这是传整个 task 进来的话
    # print (querry(task))
    query_wrap(task)
    # 如果这是传个人信息进来的话
    # print (querry2(task.get("person")))


if __name__ == '__main__':
    main()
